experiments <- c("01-adaptive_destruction_size",
"02-adaptive_best_insertion", "03-adaptive_destruction_position",
"04-adaptive_local_search", "05-adaptive_perturb", "06-adaptive_neighborhood_size")
perf_experiments <- c(experiments, "09-final_comparison")
perfs <- map(perf_experiments, ~{
exp_folder <- here("reports", "aos", "data", .x)
perf_folder <- file.path(exp_folder, "perf")
load_perfs(perf_folder) %>%
mutate(experiment = .x)
})
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
names(perfs) <- perf_experiments
extra_perfs <- c("01-adaptive_destruction_size",
"02-adaptive_best_insertion", "03-adaptive_destruction_position",
"04-adaptive_local_search", "05-adaptive_perturb", "06-adaptive_neighborhood_size", "09-final_comparison")
extras <- map(extra_perfs, ~{
exp_folder <- here("reports", "aos", "data", .x)
perf_folder <- file.path(exp_folder, "extra")
dt <- load_perfs(perf_folder) %>%
mutate(experiment = .x)
perfs[[.x]] <<- bind_rows(perfs[[.x]], dt)
dt
})
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
#>
#>
#> ── Column specification ────────────────────────────────────────────────────────
#> cols(
#> dist = col_character(),
#> corr = col_character(),
#> no_jobs = col_double(),
#> no_machines = col_double(),
#> problem = col_character(),
#> type = col_character(),
#> objective = col_character(),
#> instance = col_character(),
#> cost = col_double(),
#> time = col_double(),
#> no_evals = col_double()
#> )
names(extras) <- extra_perfs
perfs[["01-adaptive_destruction_size"]] %>%
ggplot(aes(x = algo, fill = algo, y = arfp)) +
geom_violin() +
labs(x = NULL) +
ggtitle("Adaptive destruction size") +
theme_minimal()
perfs[["01-adaptive_destruction_size"]] %>%
group_by(id) %>%
mutate(rank = rank(fitness)) %>%
select(algo, id, rank) %>%
ggplot(aes(x = algo, fill = algo, y = rank)) +
geom_violin() +
labs(x = NULL) +
ggtitle("Adaptive destruction size") +
theme_minimal()
perfs[["01-adaptive_destruction_size"]] %>%
group_by(algo) %>%
summarize(
q00 = quantile(arfp, 0.0),
q25 = quantile(arfp, 0.25),
q50 = quantile(arfp, 0.5),
mean = mean(arfp),
q75 = quantile(arfp, 0.75),
q100 = quantile(arfp, 1.00)
) %>% knitr::kable(booktabs = TRUE)
| default |
-0.2634677 |
0 |
0.2164343 |
0.5948598 |
0.9918071 |
4.804028 |
| epsilon_greedy |
-0.1425145 |
0 |
0.1785848 |
0.4913083 |
0.7952873 |
4.317461 |
| frrmab |
-0.1218769 |
0 |
0.1057885 |
0.4535496 |
0.7135609 |
4.804028 |
| linucb |
-0.1218769 |
0 |
0.1742060 |
0.4413102 |
0.7292240 |
3.904245 |
| pm |
-0.1218769 |
0 |
0.1897755 |
0.4664731 |
0.8037411 |
4.656689 |
| random-2_4 |
-0.2282447 |
0 |
0.2712795 |
0.6478915 |
1.0316041 |
5.609914 |
| random-2_4_6 |
-0.0758921 |
0 |
0.1899424 |
0.5058764 |
0.8477862 |
3.719311 |
| random-4_6 |
-0.1218769 |
0 |
0.1453131 |
0.4792023 |
0.8156606 |
4.031528 |
| random-4_8 |
-0.3411266 |
0 |
0.1386972 |
0.4156840 |
0.6595141 |
3.948798 |
| ts |
-0.2602926 |
0 |
0.1006535 |
0.4375860 |
0.6696480 |
4.804028 |
perfs[["01-adaptive_destruction_size"]] %>%
group_by(algo) %>%
summarize(
q00 = quantile(arfp, 0.0),
q25 = quantile(arfp, 0.25),
q50 = quantile(arfp, 0.5),
mean = mean(arfp),
q75 = quantile(arfp, 0.75),
q100 = quantile(arfp, 1.00)
) %>% knitr::kable(booktabs = TRUE)
| default |
-0.2634677 |
0 |
0.2164343 |
0.5948598 |
0.9918071 |
4.804028 |
| epsilon_greedy |
-0.1425145 |
0 |
0.1785848 |
0.4913083 |
0.7952873 |
4.317461 |
| frrmab |
-0.1218769 |
0 |
0.1057885 |
0.4535496 |
0.7135609 |
4.804028 |
| linucb |
-0.1218769 |
0 |
0.1742060 |
0.4413102 |
0.7292240 |
3.904245 |
| pm |
-0.1218769 |
0 |
0.1897755 |
0.4664731 |
0.8037411 |
4.656689 |
| random-2_4 |
-0.2282447 |
0 |
0.2712795 |
0.6478915 |
1.0316041 |
5.609914 |
| random-2_4_6 |
-0.0758921 |
0 |
0.1899424 |
0.5058764 |
0.8477862 |
3.719311 |
| random-4_6 |
-0.1218769 |
0 |
0.1453131 |
0.4792023 |
0.8156606 |
4.031528 |
| random-4_8 |
-0.3411266 |
0 |
0.1386972 |
0.4156840 |
0.6595141 |
3.948798 |
| ts |
-0.2602926 |
0 |
0.1006535 |
0.4375860 |
0.6696480 |
4.804028 |
algo_perfs <- perfs[["01-adaptive_destruction_size"]] %>%
filter(!(algo %in% c('random-2_4','random-2_4_6','random-4_6')))
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> default epsilon_greedy frrmab linucb pm random-4_8
#> epsilon_greedy 3.1e-11 - - - - -
#> frrmab < 2e-16 0.00044 - - - -
#> linucb 7.1e-14 0.08580 0.75125 - - -
#> pm 6.1e-14 0.71573 0.09970 0.89278 - -
#> random-4_8 < 2e-16 1.5e-07 0.71573 0.03851 0.00044 -
#> ts < 2e-16 2.2e-05 0.99571 0.33863 0.01531 0.97051
#>
#> P value adjustment method: single-step
perfs[["02-adaptive_best_insertion"]] %>%
ggplot(aes(x = algo, fill = algo, y = arfp)) +
geom_violin() +
labs(x = NULL) +
scale_y_continuous(trans = "pseudo_log") +
ggtitle("Adaptive local search focus") +
theme_minimal()
perfs[["02-adaptive_best_insertion"]] %>%
group_by(algo) %>%
summarize(
q00 = quantile(arfp, 0.0),
q25 = quantile(arfp, 0.25),
q50 = quantile(arfp, 0.5),
mean = mean(arfp),
q75 = quantile(arfp, 0.75),
q100 = quantile(arfp, 1.00)
) %>% knitr::kable(booktabs = TRUE)
| default |
-0.2634677 |
0 |
0.2164343 |
0.5948598 |
0.9918071 |
4.804028 |
| epsilon_greedy |
-0.1218769 |
0 |
0.1921976 |
0.5461791 |
0.9004010 |
5.214231 |
| frrmab |
-0.0609385 |
0 |
0.1800011 |
0.5541156 |
0.8848130 |
4.843860 |
| linucb |
-0.1218769 |
0 |
0.2087576 |
0.5696649 |
0.8966363 |
4.744624 |
| pm |
-0.1218769 |
0 |
0.2205438 |
0.5834869 |
0.9316094 |
5.772001 |
| ts |
-0.1218769 |
0 |
0.2028308 |
0.5555989 |
0.8754067 |
4.843860 |
algo_perfs <- perfs[["02-adaptive_best_insertion"]]
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> default epsilon_greedy frrmab linucb pm
#> epsilon_greedy 0.00011 - - - -
#> frrmab 4.9e-05 0.99998 - - -
#> linucb 0.02624 0.72538 0.61201 - -
#> pm 0.13486 0.33834 0.24534 0.99080 -
#> ts 0.00020 1.00000 0.99965 0.79894 0.41537
#>
#> P value adjustment method: single-step
perfs[["03-adaptive_destruction_position"]] %>%
ggplot(aes(x = algo, fill = algo, y = arfp)) +
geom_violin() +
labs(x = NULL) +
ggtitle("Adaptive destruction position") +
theme_minimal()
perfs[["03-adaptive_destruction_position"]] %>%
group_by(algo) %>%
summarize(
q00 = quantile(arfp, 0.0),
q25 = quantile(arfp, 0.25),
q50 = quantile(arfp, 0.5),
mean = mean(arfp),
q75 = quantile(arfp, 0.75),
q100 = quantile(arfp, 1.00)
) %>% knitr::kable(booktabs = TRUE)
| default |
-0.2634677 |
0.0000000 |
0.2164343 |
0.5942982 |
0.9886761 |
4.804028 |
| epsilon_greedy |
-0.1681322 |
0.0000000 |
0.2337973 |
0.5750158 |
0.9234700 |
4.732908 |
| frrmab |
-0.1451215 |
0.0000000 |
0.2023799 |
0.5530878 |
0.9248768 |
4.825145 |
| linucb |
-0.1218769 |
0.0053988 |
0.3602194 |
0.6723328 |
1.1170163 |
5.175201 |
| pm |
-0.1218769 |
0.0000000 |
0.2350556 |
0.5632123 |
0.9339044 |
4.825145 |
| ts |
-0.1218769 |
0.0000000 |
0.2144830 |
0.5480395 |
0.8876607 |
5.164588 |
algo_perfs <- perfs[["03-adaptive_destruction_position"]]
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> default epsilon_greedy frrmab linucb pm
#> epsilon_greedy 0.1467 - - - -
#> frrmab 0.0039 0.8363 - - -
#> linucb 0.0040 2.2e-08 6.2e-12 - -
#> pm 0.0976 1.0000 0.9077 7.4e-09 -
#> ts 0.0093 0.9311 0.9999 3.6e-11 0.9698
#>
#> P value adjustment method: single-step
perfs[["04-adaptive_local_search"]] %>%
ggplot(aes(x = algo, fill = algo, y = arfp)) +
geom_violin() +
labs(x = NULL) +
ggtitle("Adaptive local search") +
theme_minimal()
perfs[["04-adaptive_local_search"]] %>%
group_by(algo) %>%
summarize(
q00 = quantile(arfp, 0.0),
q25 = quantile(arfp, 0.25),
q50 = quantile(arfp, 0.5),
mean = mean(arfp),
q75 = quantile(arfp, 0.75),
q100 = quantile(arfp, 1.00)
) %>% knitr::kable(booktabs = TRUE)
| default |
-0.2634677 |
0 |
0.2164343 |
0.5948598 |
0.9918071 |
4.804028 |
| epsilon_greedy |
-0.1218769 |
0 |
0.2435542 |
0.6860770 |
1.0222437 |
6.712545 |
| frrmab |
-0.1821084 |
0 |
0.2321387 |
0.6662321 |
1.0314004 |
6.195962 |
| linucb |
-0.1218769 |
0 |
0.2580531 |
0.6963908 |
1.0537408 |
6.617857 |
| pm |
-0.1730310 |
0 |
0.2321169 |
0.6858710 |
1.0607713 |
6.203405 |
| random |
-0.1218769 |
0 |
0.2525543 |
0.6941423 |
1.0851414 |
6.203405 |
| ts |
-0.1218769 |
0 |
0.2345952 |
0.6621848 |
1.0453204 |
5.473182 |
algo_perfs <- perfs[["04-adaptive_local_search"]]
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> default epsilon_greedy frrmab linucb pm random
#> epsilon_greedy 0.75 - - - - -
#> frrmab 0.29 0.99 - - - -
#> linucb 0.12 0.92 1.00 - - -
#> pm 0.35 1.00 1.00 1.00 - -
#> random 0.29 0.99 1.00 1.00 1.00 -
#> ts 0.87 1.00 0.97 0.82 0.98 0.96
#>
#> P value adjustment method: single-step
perfs[["05-adaptive_perturb"]] %>%
ggplot(aes(x = algo, fill = algo, y = arfp)) +
geom_violin() +
labs(x = NULL) +
scale_y_continuous(trans = "pseudo_log") +
ggtitle("Adaptive perturbation") +
theme_minimal()
perfs[["05-adaptive_perturb"]] %>%
group_by(algo) %>%
summarize(
q00 = quantile(arfp, 0.0),
q25 = quantile(arfp, 0.25),
q50 = quantile(arfp, 0.5),
mean = mean(arfp),
q75 = quantile(arfp, 0.75),
q100 = quantile(arfp, 1.00)
) %>% knitr::kable(booktabs = TRUE)
| default |
-0.2634677 |
0 |
0.2154749 |
0.5952273 |
0.9886761 |
4.804028 |
| epsilon_greedy |
-0.5071793 |
0 |
0.0620930 |
0.3774206 |
0.5734283 |
3.808214 |
| frrmab |
-0.3199051 |
0 |
0.0486414 |
0.3529366 |
0.5083178 |
4.237827 |
| linucb |
-0.3199936 |
0 |
0.0978909 |
0.4340297 |
0.6175167 |
4.804028 |
| pm |
-0.7400194 |
0 |
0.0474168 |
0.3511759 |
0.5011281 |
4.426239 |
| random |
-0.7405741 |
0 |
0.0459770 |
0.3482407 |
0.4987108 |
4.426239 |
| ts |
-0.5179541 |
0 |
0.0456852 |
0.3434712 |
0.4676016 |
4.940754 |
algo_perfs <- perfs[["05-adaptive_perturb"]]
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> default epsilon_greedy frrmab linucb pm random
#> epsilon_greedy 6.7e-14 - - - - -
#> frrmab < 2e-16 0.27973 - - - -
#> linucb 5.1e-14 0.53636 0.00102 - - -
#> pm < 2e-16 0.16353 0.99998 0.00034 - -
#> random < 2e-16 0.12781 0.99982 0.00021 1.00000 -
#> ts < 2e-16 0.45106 0.99996 0.00321 0.99817 0.99509
#>
#> P value adjustment method: single-step
perfs[["06-adaptive_neighborhood_size"]] %>%
ggplot(aes(x = algo, fill = algo, y = arfp)) +
geom_violin() +
labs(x = NULL) +
ggtitle("Adaptive neighborhood size") +
theme_minimal()
perfs[["06-adaptive_neighborhood_size"]] %>%
group_by(algo) %>%
summarize(
q00 = quantile(arfp, 0.0),
q25 = quantile(arfp, 0.25),
q50 = quantile(arfp, 0.5),
mean = mean(arfp),
q75 = quantile(arfp, 0.75),
q100 = quantile(arfp, 1.00)
) %>% knitr::kable(booktabs = TRUE)
| default |
-0.2634677 |
0 |
0.2190707 |
0.5959512 |
0.9886761 |
4.804028 |
| epsilon_greedy |
-0.0882223 |
0 |
0.2190367 |
0.5986173 |
0.9801334 |
4.276694 |
| frrmab |
-0.1218769 |
0 |
0.1964958 |
0.5892183 |
1.0078721 |
4.344925 |
| linucb |
-0.1098014 |
0 |
0.1903865 |
0.5886529 |
0.9714045 |
4.349643 |
| pm |
-0.1218769 |
0 |
0.2308222 |
0.5929648 |
0.9750639 |
4.235910 |
| random-10 |
-0.1218769 |
0 |
0.2471588 |
0.6459067 |
1.0161432 |
5.165043 |
| random-2 |
-0.1218769 |
0 |
0.2188905 |
0.5665911 |
0.9406860 |
5.010496 |
| random-3 |
-0.1218769 |
0 |
0.2201530 |
0.5902228 |
0.9615418 |
4.235910 |
| random-5 |
-0.1218769 |
0 |
0.2222259 |
0.6046689 |
1.0474191 |
4.772052 |
| ts |
-0.0882223 |
0 |
0.2185640 |
0.5977707 |
0.9728888 |
4.344925 |
algo_perfs <- perfs[["06-adaptive_neighborhood_size"]] %>%
filter(!(algo %in% c('random-3', 'random-5', 'random-10')))
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> default epsilon_greedy frrmab linucb pm random-2
#> epsilon_greedy 0.136 - - - - -
#> frrmab 0.230 1.000 - - - -
#> linucb 0.303 1.000 1.000 - - -
#> pm 0.644 0.971 0.994 0.998 - -
#> random-2 0.038 0.999 0.992 0.979 0.811 -
#> ts 0.729 0.945 0.985 0.995 1.000 0.735
#>
#> P value adjustment method: single-step
perfs %>%
bind_rows() %>%
group_by(experiment, algo) %>%
summarise(
arfp = sprintf("%.3f %.2f", mean(arfp), sd(arfp))
) %>%
pivot_wider(names_from = "algo", values_from = "arfp") %>%
knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment'. You can override using the `.groups` argument.
| 01-adaptive_destruction_size |
0.595 0.80 |
0.491 0.67 |
0.454 0.69 |
0.441 0.61 |
0.466 0.64 |
0.648 0.84 |
0.506 0.68 |
0.479 0.67 |
0.416 0.60 |
0.438 0.67 |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
| 02-adaptive_best_insertion |
0.595 0.80 |
0.546 0.74 |
0.554 0.76 |
0.570 0.76 |
0.583 0.80 |
NA |
NA |
NA |
NA |
0.556 0.76 |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
| 03-adaptive_destruction_position |
0.594 0.80 |
0.575 0.75 |
0.553 0.73 |
0.672 0.79 |
0.563 0.73 |
NA |
NA |
NA |
NA |
0.548 0.73 |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
| 04-adaptive_local_search |
0.595 0.80 |
0.686 1.00 |
0.666 0.94 |
0.696 1.00 |
0.686 1.00 |
NA |
NA |
NA |
NA |
0.662 0.96 |
0.694 1.00 |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
| 05-adaptive_perturb |
0.595 0.80 |
0.377 0.62 |
0.353 0.61 |
0.434 0.70 |
0.351 0.63 |
NA |
NA |
NA |
NA |
0.343 0.63 |
0.348 0.63 |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
| 06-adaptive_neighborhood_size |
0.596 0.80 |
0.599 0.80 |
0.589 0.80 |
0.589 0.80 |
0.593 0.79 |
NA |
NA |
NA |
NA |
0.598 0.81 |
NA |
0.646 0.86 |
0.567 0.75 |
0.590 0.79 |
0.605 0.79 |
NA |
NA |
NA |
| 09-final_comparison |
0.595 0.80 |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
NA |
0.455 0.77 |
0.380 0.64 |
0.343 0.63 |
filter_ig_default <- function(config) {
remove_param <- c(
"IG.AdaptivePosition.AOS.WarmUp.Strategy",
# "IG.AdaptivePosition.AOS.Options",
"IG.AdaptivePosition.AOS.Strategy",
"IG.AdaptiveBestInsertion.AOS.Strategy",
"IG.AdaptivePerturb.AOS.Strategy",
"IG.AdaptiveLocalSearch.AOS.Strategy",
"IG.AdaptiveNeighborhoodSize.AOS.Strategy",
"IG.AOS.Strategy"
)
config[(str_detect(names(config), "AOS") |
str_detect(names(config), "AdaptiveBestInsertion") |
str_detect(names(config), "AdaptivePosition")) &
!(names(config) %in% remove_param)]
}
format_config <- function(config) {
param_names <- names(config) %>%
str_replace("PM\\.RewardType", "PMRewardType") %>%
str_remove("^IG\\.(AdaptivePosition|AdaptiveLocalSearch|AdaptiveBestInsertion|AdaptiveNeighborhoodSize|AdaptivePerturb)\\.AOS\\.") %>%
str_remove("^IG\\.AOS\\.") %>%
str_remove("^IG\\.AdaptiveBestInsertion\\.") %>%
str_remove("^IG\\.AdaptivePosition\\.") %>%
str_remove("^(PM|FRRMAB|LINUCB|TS)\\.")
paste(param_names, config, sep = " = ")
}
load_config <- function(config_path) {
read_rds(config_path)[1,] %>%
df_to_character() %>%
filter_ig_default() %>%
format_config()
}
load_configs <- function(irace_folder, path = "all") {
tibble(
path = path,
full_path = file.path(irace_folder, path)
) %>%
select(path, full_path) %>%
mutate(
configs = map(full_path, function(path) {
tibble(algo = dir(path)) %>%
filter(!str_detect(algo, "_log|_rec")) %>%
mutate(config = map(file.path(path, algo), load_config))
})
) %>%
unnest(configs)
}
configs_tidy <- map_dfr(experiments, ~{
here("reports", "aos", "data", .x) %>%
file.path("irace") %>%
load_configs() %>%
mutate(exp = .x) %>%
select(exp, algo, config)
}) %>%
unnest(config) %>%
separate(config, c("param", "value"), sep = " = ")
configs_tidy %>%
filter((param %in% c("RewardType"))) %>%
filter(!(param %in% c("WarmUp.Strategy"))) %>% # pivot_wider(names_from = "param", values_from = "value") %>%
arrange(exp, algo, param) %>%
mmtable(value) +
header_top(algo) +
header_left(exp)
| |
|
|
|
|
|
|
igepsilon_greedy.rds |
igfrrmab.rds |
iglinucb.rds |
igpm.rds |
igts.rds |
| 01-adaptive_destruction_size |
0 |
1 |
2 |
1 |
2 |
| 03-adaptive_destruction_position |
2 |
3 |
3 |
0 |
1 |
| 04-adaptive_local_search |
2 |
2 |
2 |
0 |
1 |
| 05-adaptive_perturb |
3 |
1 |
3 |
0 |
1 |
| 06-adaptive_neighborhood_size |
1 |
0 |
0 |
3 |
2 |
configs_tidy %>%
filter((param %in% c("Options"))) %>%
filter(!(param %in% c("WarmUp.Strategy"))) %>% # pivot_wider(names_from = "param", values_from = "value") %>%
arrange(exp, algo, param) %>%
mmtable(value) +
header_top(algo) +
header_left(exp)
| |
|
|
|
|
|
|
igepsilon_greedy.rds |
igfrrmab.rds |
iglinucb.rds |
igpm.rds |
igts.rds |
| 01-adaptive_destruction_size |
2_4_6 |
4_8 |
4_8 |
4_6 |
4_8 |
configs_tidy %>%
filter((param %in% c("NoArms"))) %>%
filter(!(param %in% c("WarmUp.Strategy"))) %>% # pivot_wider(names_from = "param", values_from = "value") %>%
arrange(exp, algo, param) %>%
mmtable(value) +
header_top(algo) +
header_left(exp)
| |
|
|
|
|
|
|
igepsilon_greedy.rds |
igfrrmab.rds |
iglinucb.rds |
igpm.rds |
igts.rds |
| 02-adaptive_best_insertion |
fixed_3 |
fixed_10 |
fixed_3 |
no_jobs |
fixed_10 |
| 03-adaptive_destruction_position |
fixed_3 |
fixed_3 |
fixed_3 |
fixed_3 |
fixed_10 |
| 06-adaptive_neighborhood_size |
3 |
2 |
2 |
3 |
2 |
configs_tidy %>%
filter((param %in% c("WindowSize", "UpdateWindow", "C"))) %>%
filter(!(param %in% c("WarmUp.Strategy"))) %>% # pivot_wider(names_from = "param", values_from = "value") %>%
arrange(exp, algo, param) %>%
mmtable(value) +
header_top(algo) +
header_left(exp)
| |
|
|
|
|
igfrrmab.rds |
igpm.rds |
igts.rds |
| 01-adaptive_destruction_size |
406 |
128 |
NA |
| 02-adaptive_best_insertion |
271 |
125 |
299 |
| 03-adaptive_destruction_position |
350 |
357 |
NA |
| 04-adaptive_local_search |
188 |
283 |
414 |
| 05-adaptive_perturb |
59 |
283 |
21 |
| 06-adaptive_neighborhood_size |
32 |
340 |
358 |
extras[[1]] %>%
ggplot(aes(x = algo, fill = algo, y = arfp)) +
geom_violin() +
labs(x = NULL) +
scale_y_continuous(trans = "pseudo_log") +
ggtitle("Adaptive perturb-ds") +
theme_minimal()
extras[[1]] %>%
bind_rows() %>%
group_by(experiment, algo) %>%
summarise(
arfp = sprintf("%.3f (%.3f)", mean(arfp), sd(arfp))
) %>%
pivot_wider(names_from = "algo", values_from = "arfp") %>%
knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment'. You can override using the `.groups` argument.
| 01-adaptive_destruction_size |
0.938 (0.883) |
0.761 (0.739) |
0.702 (0.760) |
0.685 (0.661) |
0.720 (0.708) |
0.985 (0.949) |
0.804 (0.748) |
0.757 (0.739) |
0.648 (0.683) |
0.686 (0.767) |
algo_perfs <- perfs[["09-final_comparison"]]
algo_perfs %>%
ggplot(aes(x = algo, fill = algo, y = arfp)) +
geom_violin() +
labs(x = NULL) +
scale_y_continuous(trans = "pseudo_log") +
ggtitle("Adaptive perturb-ds") +
theme_minimal()
frdAllPairsNemenyiTest(algo_perfs$arfp, algo_perfs$algo, paste(algo_perfs$id, "-", algo_perfs$seed))
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> adapt-all-arpd-final adapt-no-local-search-final
#> adapt-no-local-search-final 0.00072 -
#> default 1.2e-09 3.5e-14
#> ts-perturb 1.7e-09 0.06388
#> default
#> adapt-no-local-search-final -
#> default -
#> ts-perturb < 2e-16
#>
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
bind_rows() %>%
group_by(experiment, algo) %>%
summarise(
arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
) %>%
pivot_wider(names_from = "algo", values_from = "arfp") %>%
knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment'. You can override using the `.groups` argument.
| 09-final_comparison |
0.455 (0.77) |
0.380 (0.64) |
0.595 (0.80) |
0.343 (0.63) |
frdAllPairsNemenyiTest(perfs[["09-final_comparison"]]$arfp, perfs[["09-final_comparison"]]$algo, paste(perfs[["09-final_comparison"]]$id, "-", perfs[["09-final_comparison"]]$seed))
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> adapt-all-arpd-final adapt-no-local-search-final
#> adapt-no-local-search-final 0.00072 -
#> default 1.2e-09 3.5e-14
#> ts-perturb 1.7e-09 0.06388
#> default
#> adapt-no-local-search-final -
#> default -
#> ts-perturb < 2e-16
#>
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
bind_rows() %>%
group_by(experiment, algo, objective) %>%
summarise(
arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
) %>%
pivot_wider(names_from = "algo", values_from = "arfp") %>%
knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment', 'algo'. You can override using the `.groups` argument.
| 09-final_comparison |
FLOWTIME |
0.592 (0.95) |
0.466 (0.80) |
0.763 (0.95) |
0.436 (0.78) |
| 09-final_comparison |
MAKESPAN |
0.319 (0.49) |
0.294 (0.42) |
0.426 (0.56) |
0.251 (0.40) |
test_by_group <- function(perfs, group_var) {
perfs %>%
group_by(across(group_var)) %>%
group_split(.keep = TRUE) %>%
map(~frdAllPairsNemenyiTest(.x$arfp, paste(.x$algo, .x[[group_var]]), paste(.x$id, "-", .x$seed)))
}
test_by_group(perfs[["09-final_comparison"]], 'objective')
#> Note: Using an external vector in selections is ambiguous.
#> ℹ Use `all_of(group_var)` instead of `group_var` to silence this message.
#> ℹ See <https://tidyselect.r-lib.org/reference/faq-external-vector.html>.
#> This message is displayed once per session.
#> [[1]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> adapt-all-arpd-final FLOWTIME
#> adapt-no-local-search-final FLOWTIME 0.001
#> default FLOWTIME 7.7e-08
#> ts-perturb FLOWTIME 2.8e-06
#> adapt-no-local-search-final FLOWTIME
#> adapt-no-local-search-final FLOWTIME -
#> default FLOWTIME 3.6e-14
#> ts-perturb FLOWTIME 0.571
#> default FLOWTIME
#> adapt-no-local-search-final FLOWTIME -
#> default FLOWTIME -
#> ts-perturb FLOWTIME 4.0e-14
#>
#> P value adjustment method: single-step
#>
#> [[2]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#>
#> data: y, groups and blocks
#> adapt-all-arpd-final MAKESPAN
#> adapt-no-local-search-final MAKESPAN 0.33694
#> default MAKESPAN 0.00512
#> ts-perturb MAKESPAN 0.00059
#> adapt-no-local-search-final MAKESPAN
#> adapt-no-local-search-final MAKESPAN -
#> default MAKESPAN 3.6e-06
#> ts-perturb MAKESPAN 0.11979
#> default MAKESPAN
#> adapt-no-local-search-final MAKESPAN -
#> default MAKESPAN -
#> ts-perturb MAKESPAN 3.6e-12
#>
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
bind_rows() %>%
group_by(experiment, algo, type) %>%
summarise(
arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
) %>%
pivot_wider(names_from = "algo", values_from = "arfp") %>%
knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment', 'algo'. You can override using the `.groups` argument.
| 09-final_comparison |
NOIDLE |
0.888 (1.07) |
0.759 (0.90) |
0.793 (0.92) |
0.708 (0.90) |
| 09-final_comparison |
NOWAIT |
0.144 (0.26) |
0.136 (0.25) |
0.592 (0.76) |
0.106 (0.22) |
| 09-final_comparison |
PERM |
0.334 (0.50) |
0.245 (0.39) |
0.400 (0.63) |
0.216 (0.35) |
test_by_group(perfs[["09-final_comparison"]], 'type')
#> [[1]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> adapt-all-arpd-final NOIDLE
#> adapt-no-local-search-final NOIDLE 0.01733
#> default NOIDLE 0.94203
#> ts-perturb NOIDLE 5e-05
#> adapt-no-local-search-final NOIDLE
#> adapt-no-local-search-final NOIDLE -
#> default NOIDLE 0.08233
#> ts-perturb NOIDLE 0.42803
#> default NOIDLE
#> adapt-no-local-search-final NOIDLE -
#> default NOIDLE -
#> ts-perturb NOIDLE 0.00058
#>
#> P value adjustment method: single-step
#>
#> [[2]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#>
#> data: y, groups and blocks
#> adapt-all-arpd-final NOWAIT
#> adapt-no-local-search-final NOWAIT 0.84
#> default NOWAIT 2.9e-14
#> ts-perturb NOWAIT 0.12
#> adapt-no-local-search-final NOWAIT
#> adapt-no-local-search-final NOWAIT -
#> default NOWAIT 3.7e-14
#> ts-perturb NOWAIT 0.51
#> default NOWAIT
#> adapt-no-local-search-final NOWAIT -
#> default NOWAIT -
#> ts-perturb NOWAIT < 2e-16
#>
#> P value adjustment method: single-step
#>
#> [[3]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#>
#> data: y, groups and blocks
#> adapt-all-arpd-final PERM
#> adapt-no-local-search-final PERM 0.02086
#> default PERM 0.20959
#> ts-perturb PERM 0.00011
#> adapt-no-local-search-final PERM default PERM
#> adapt-no-local-search-final PERM - -
#> default PERM 8.4e-06 -
#> ts-perturb PERM 0.50361 3.1e-09
#>
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
bind_rows() %>%
group_by(experiment, algo, dist, corr) %>%
summarise(
arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
) %>%
pivot_wider(names_from = "algo", values_from = "arfp") %>%
knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment', 'algo', 'dist'. You can override using the `.groups` argument.
| 09-final_comparison |
exponential |
random |
0.805 (1.05) |
0.614 (0.82) |
1.093 (0.99) |
0.614 (0.84) |
| 09-final_comparison |
uniform |
job-correlated |
0.317 (0.52) |
0.286 (0.50) |
0.403 (0.53) |
0.269 (0.48) |
| 09-final_comparison |
uniform |
machine-correlated |
0.042 (0.08) |
0.031 (0.06) |
0.070 (0.13) |
0.030 (0.07) |
| 09-final_comparison |
uniform |
random |
0.658 (0.78) |
0.589 (0.72) |
0.814 (0.81) |
0.461 (0.67) |
test_by_group(perfs[["09-final_comparison"]] %>% mutate(dist_corr = paste(dist, corr)), 'dist_corr')
#> [[1]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> adapt-all-arpd-final exponential random
#> adapt-no-local-search-final exponential random 0.0158
#> default exponential random 0.0107
#> ts-perturb exponential random 0.0052
#> adapt-no-local-search-final exponential random
#> adapt-no-local-search-final exponential random -
#> default exponential random 8.0e-09
#> ts-perturb exponential random 0.9870
#> default exponential random
#> adapt-no-local-search-final exponential random -
#> default exponential random -
#> ts-perturb exponential random 9.4e-10
#>
#> P value adjustment method: single-step
#>
#> [[2]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#>
#> data: y, groups and blocks
#> adapt-all-arpd-final uniform job-correlated
#> adapt-no-local-search-final uniform job-correlated 0.8799
#> default uniform job-correlated 0.0003
#> ts-perturb uniform job-correlated 0.2024
#> adapt-no-local-search-final uniform job-correlated
#> adapt-no-local-search-final uniform job-correlated -
#> default uniform job-correlated 9.9e-06
#> ts-perturb uniform job-correlated 0.6142
#> default uniform job-correlated
#> adapt-no-local-search-final uniform job-correlated -
#> default uniform job-correlated -
#> ts-perturb uniform job-correlated 1.1e-08
#>
#> P value adjustment method: single-step
#>
#> [[3]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#>
#> data: y, groups and blocks
#> adapt-all-arpd-final uniform machine-correlated
#> adapt-no-local-search-final uniform machine-correlated 0.1355
#> default uniform machine-correlated 0.0052
#> ts-perturb uniform machine-correlated 0.0730
#> adapt-no-local-search-final uniform machine-correlated
#> adapt-no-local-search-final uniform machine-correlated -
#> default uniform machine-correlated 2.8e-07
#> ts-perturb uniform machine-correlated 0.9935
#> default uniform machine-correlated
#> adapt-no-local-search-final uniform machine-correlated -
#> default uniform machine-correlated -
#> ts-perturb uniform machine-correlated 6.1e-08
#>
#> P value adjustment method: single-step
#>
#> [[4]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#>
#> data: y, groups and blocks
#> adapt-all-arpd-final uniform random
#> adapt-no-local-search-final uniform random 0.27178
#> default uniform random 0.10258
#> ts-perturb uniform random 4.9e-06
#> adapt-no-local-search-final uniform random
#> adapt-no-local-search-final uniform random -
#> default uniform random 0.00026
#> ts-perturb uniform random 0.00950
#> default uniform random
#> adapt-no-local-search-final uniform random -
#> default uniform random -
#> ts-perturb uniform random 3.3e-12
#>
#> P value adjustment method: single-step
perfs[["09-final_comparison"]] %>%
bind_rows() %>%
group_by(experiment, algo, no_jobs<=50) %>%
summarise(
arfp = sprintf("%.3f (%.2f)", mean(arfp), sd(arfp))
) %>%
pivot_wider(names_from = "algo", values_from = "arfp") %>%
knitr::kable(booktabs = TRUE)
#> `summarise()` has grouped output by 'experiment', 'algo'. You can override using the `.groups` argument.
| 09-final_comparison |
FALSE |
0.697 (0.89) |
0.554 (0.76) |
0.938 (0.88) |
0.515 (0.75) |
| 09-final_comparison |
TRUE |
0.214 (0.52) |
0.206 (0.45) |
0.251 (0.50) |
0.172 (0.39) |
test_by_group(perfs[["09-final_comparison"]] %>% mutate(no_jobs = no_jobs <= 50), 'no_jobs')
#> [[1]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#> data: y, groups and blocks
#> adapt-all-arpd-final FALSE
#> adapt-no-local-search-final FALSE 3.4e-08
#> default FALSE 1.1e-09
#> ts-perturb FALSE 1.5e-13
#> adapt-no-local-search-final FALSE
#> adapt-no-local-search-final FALSE -
#> default FALSE < 2e-16
#> ts-perturb FALSE 0.26
#> default FALSE
#> adapt-no-local-search-final FALSE -
#> default FALSE -
#> ts-perturb FALSE < 2e-16
#>
#> P value adjustment method: single-step
#>
#> [[2]]
#>
#> Pairwise comparisons using Nemenyi-Wilcoxon-Wilcox all-pairs test for a two-way balanced complete block design
#>
#> data: y, groups and blocks
#> adapt-all-arpd-final TRUE
#> adapt-no-local-search-final TRUE 0.97834
#> default TRUE 0.04302
#> ts-perturb TRUE 0.57891
#> adapt-no-local-search-final TRUE default TRUE
#> adapt-no-local-search-final TRUE - -
#> default TRUE 0.11646 -
#> ts-perturb TRUE 0.33694 0.00056
#>
#> P value adjustment method: single-step